import pandas as pd
import numpy as np
from jili.core import load, save
from jili.data.db import getdb_client, insert_one, get_calender_a
from jili.calc.graph import graph
from jili.calc import forwardwindows
from jili.tool.convert import str2datetime
import os, datetime
from tqdm import tqdm
from jili.tool.state import config


def getfactor_bydate(url, objs, factors, date):
    date0 = str2datetime(date)
    t = {}
    for i in factors:
        url0 = os.path.join(url, i)
        a1 = load(url0)
        print(i, len(a1))
        a = a1[date0]
        for obj in objs:
            if obj not in t.keys():
                t[obj] = {}
            if obj in a.keys():
                tt = a[obj]
            else:
                tt = None
            if isinstance(tt, dict):
                t[obj].update(tt)
            else:
                t[obj][i] = tt
    td = pd.DataFrame(t).T
    return td

class fullend_date_calcor:
    def __init__(self, key="obj"):
        self.ann_end_date = {}
        self.lastend_date = {}
        self.key = key
        self.ll={}
        self.end_ann_date={}

    def ondata(self, i):
        key = i[self.key]
        ann_date = i["ann_date"]
        end_date = i["end_date"]
        if end_date[-4:] in ["1231","0630"]:
            if key not in self.ann_end_date.keys():
                self.ann_end_date[key] = {}
                self.end_ann_date[key]={}
            if end_date not in self.end_ann_date[key].keys():
                self.end_ann_date[key][end_date]=[]
            if ann_date not in self.end_ann_date[key][end_date]:
                self.end_ann_date[key][end_date].append(ann_date)
            if ann_date not in self.ann_end_date[key].keys():
                self.ann_end_date[key][ann_date] = []
            if end_date not in self.ann_end_date[key][ann_date]:
                self.ann_end_date[key][ann_date].append(end_date)
    def get_anndates_byenddate(self,k,end_date):
        if k in self.end_ann_date.keys():
            if end_date in self.end_ann_date[k].keys():
                return self.end_ann_date[k][end_date]
            else:
                return []
        else:
            return []

    def calc_lastend_date(self):
        for obj, v in self.ann_end_date.items():
            self.lastend_date[obj] = {}
            tt = []
            l = list(v.keys())
            l.sort()
            for ann_date in l:
                v1 = v[ann_date]
                for i in v1:
                    if i not in tt:
                        tt.append(i)
                self.lastend_date[obj][ann_date] = max(tt)
        for obj in self.lastend_date.keys():
            v=list(self.lastend_date[obj].keys())
            v.sort()
            self.ll[obj]=v

    def get_lastend_date(self, obj, ann_date):
        if not self.lastend_date:
            self.calc_lastend_date()
        if obj in self.lastend_date.keys():
            if ann_date in self.lastend_date[obj].keys():
                return self.lastend_date[obj][ann_date]
            else:
                ll=self.ll[obj]
                last_ann_date=None
                for i in ll:
                    if i<=ann_date:
                        last_ann_date=i
                    else:
                        break
                if last_ann_date:
                    return self.lastend_date[obj][last_ann_date]
                else:
                    return None
        else:
            return None


class end_date_calcor:
    def __init__(self, key="obj"):
        self.ann_end_date = {}
        self.lastend_date = {}
        self.key = key
        self.ll={}

    def ondata(self, i):
        key = i[self.key]
        ann_date = i["ann_date"]
        end_date = i["end_date"]
        if key not in self.ann_end_date.keys():
            self.ann_end_date[key] = {}
        if ann_date not in self.ann_end_date[key].keys():
            self.ann_end_date[key][ann_date] = []
        if end_date not in self.ann_end_date[key][ann_date]:
            self.ann_end_date[key][ann_date].append(end_date)

    def calc_lastend_date(self):
        for obj, v in self.ann_end_date.items():
            self.lastend_date[obj] = {}
            tt = []
            l = list(v.keys())
            l.sort()
            for ann_date in l:
                v1 = v[ann_date]
                for i in v1:
                    if i not in tt:
                        tt.append(i)
                self.lastend_date[obj][ann_date] = max(tt)
        for obj in self.lastend_date.keys():
            v=list(self.lastend_date[obj].keys())
            v.sort()
            self.ll[obj]=v

    def get_lastend_date(self, obj, ann_date):
        if not self.lastend_date:
            self.calc_lastend_date()
        if obj in self.lastend_date.keys():
            if ann_date in self.lastend_date[obj].keys():
                return self.lastend_date[obj][ann_date]
            else:
                ll=self.ll[obj]
                last_ann_date=None
                for i in ll:
                    if i<=ann_date:
                        last_ann_date=i
                    else:
                        break
                if last_ann_date:
                    return self.lastend_date[obj][last_ann_date]
                else:
                    return None
        else:
            return None

def convert_byobj2bydate(data):
    rst={}
    for obj,v in data.items():
        for date,vv in v.items():
            date1=str2datetime(date)
            if date1 not in rst.keys():
                rst[date1]={}
            rst[date1][obj]=vv
    return rst
def gen_tradedata_a(data, end_date,start_date=None):
    rst = {}
    for obj, v in tqdm(data.items()):
        if isinstance(end_date,dict):
            end_date0=end_date[obj]
        else:
            end_date0=end_date
        for t in forwardwindows(list(v.keys()), 2):
            if len(t) == 2:
                t1 = t[0]
                t2 = t[1]
            else:
                t1 = t[0]
                t2 = str2datetime(end_date0)
            tt = v[t1]
            if start_date is None:
                for date in get_calender_a(t1, t2):
                    if date not in rst.keys():
                        rst[date] = {}
                    if obj not in rst[date].keys():
                        rst[date][obj] = tt
            else:
                if start_date>=t1 and start_date<=t2:
                    for date in get_calender_a(start_date, t2):
                        if date not in rst.keys():
                            rst[date] = {}
                        if obj not in rst[date].keys():
                            rst[date][obj] = tt
                elif start_date<t1:
                    for date in get_calender_a(t1, t2):
                        if date not in rst.keys():
                            rst[date] = {}
                        if obj not in rst[date].keys():
                            rst[date][obj] = tt
    return rst
def gen_tradedata_a_noobj(data,start_date, end_date):
    rst = {}
    for t in forwardwindows(list(data.keys()), 2):
        if len(t) == 2:
            t1 = t[0]
            t2 = t[1]
        else:
            t1 = t[0]
            t2 = str2datetime(end_date)
        tt = data[t1]
        for date in get_calender_a(t1, t2):
            rst[date] = tt
    rst1={}
    if start_date:
        start=str2datetime(start_date)
        for date,v in rst.items():
            if date>=start:
                rst1[date]=v
    return rst1

class factordata_f:
    """
    0.可作为因子实例，也可以作为因子获取方法
    1.当作为因子实例时支持缓存，当作为 因子方法时不支持缓存
    2.自动判断是否按日志还是全部数据，优先使用按日期存储的数据
    3.支持因子保存功能，save方法，已经有全部转日志的方法
    期货因子  保存格式  /url/pcode/pcode_yyyymm.pkl,支持按obj获取但是最终转成pcode
    """
    def __init__(self, pcode="", url=r"G:\factor\k1d", freq="%Y%m"):
        self.pcode = pcode
        self.url = url
        self.data = {}
        self.freq = freq
    def get_arg(self,pcode,url,freq):
        if pcode is None:
            pcode=self.pcode
        if url is None:
            url=self.url
        if freq is None:
            freq=self.freq
        return pcode,url,freq
    def merge_data(self,data,freq="%Y%m"):
        data0={}
        for i in data:
            for date,v in i.items():
                key = date.strftime(freq)
                if key not in data0.keys():
                    data0[key]={}
                    data0[key][date]=v
                else:
                    if date in data0[key].keys():
                        data0[key][date].update(v)
                    else:
                        data0[key][date]=v
        return data0
    def save_all(self,data,pcode=None,url=None,freq="%Y%m"):
        pcode, url, freq = self.get_arg(pcode, url, freq)
        if data:
            data0 = {}
            if isinstance(data,dict):
                for date, v in data.items():
                    key = date.strftime(freq)
                    if key not in data0.keys():
                        data0[key] = {}
                    data0[key][date] = v
            elif isinstance(data,list):
                data0=self.merge_data(data,freq)
            for key, v in data0.items():
                save_url = os.path.join(url, pcode, pcode+"_"+key + ".pkl")
                save(v, save_url)
    def save_bydate(self,date,data,pcode=None,url=None,freq="%Y%m",isupdatepart=True):
        pcode, url, freq = self.get_arg(pcode, url, freq)
        key=date.strftime(freq)
        save_url = os.path.join(url, pcode, pcode+"_"+key + ".pkl")
        if os.path.exists(save_url):
            a = load(save_url)
            if isupdatepart:
                a[date].update(data)
            else:
                a[date]=data
            save(a, save_url)
        else:
            data1={}
            data1[date]=data
            save(data1, save_url)

    def save_new_factordata(self, data,pcode=None, url=None, freq="%Y%m",isupdatepart=True):
        pcode, url, freq = self.get_arg(pcode, url, freq)
        if data:
            data1 = {}
            if isinstance(data,dict):
                for date, v in data.items():
                    key = date.strftime(freq)
                    if key not in data1.keys():
                        data1[key] = {}
                    data1[key][date] = v
            elif isinstance(data,list):
                data1=self.merge_data(data,freq)
            for key, v in data1.items():
                save_url = os.path.join(url, pcode, pcode+"_"+key + ".pkl")
                if os.path.exists(save_url):
                    a = load(save_url)
                    if isupdatepart:
                        for d,vv in v.items():
                            a[d].update(vv)
                    else:
                        a.update(v)
                    save(a, save_url)
                else:
                    save(v, save_url)

    def get_all(self,pcode=None,url=None,freq=None):
        pcode,url,freq=self.get_arg(pcode,url,freq)
        d={}
        if pcode is None:#判断是否缓存
            if  self.data:
                d=self.data
        if not d:
            base_url = os.path.join(url, pcode)
            l=os.listdir(base_url)
            l.sort()
            for i in l:
                data_url=os.path.join(base_url,i)
                d.update(load(data_url))
            if pcode is None:
                self.data=d
        return  d

    def check_data(self, url1, url2, start_date, end_date):
        print(url1, url2, start_date, end_date)
        data1 = {}
        if url1[-4:] == ".pkl":
            data1 = load(url1)
        else:
            for date, v in self.iter_data(start=start_date, end=end_date, url=url1):
                data1[date] = v
        data2 = {}
        if url2[-4:] == ".pkl":
            data2 = load(url2)
        else:
            for date, v in self.iter_data(start=start_date, end=end_date, url=url2):
                data2[date] = v
        print("检查是否缺失数据,data1",len(data1),"data2",len(data2))
        if not data1:
            print("数据源一，没有数据", data1)
            return False
        if not data2:
            print("数据源二，没有数据", data2)
            return False
        for date in data1.keys():
            if date not in data2.keys():
                print("第二源比第一源 缺少日期", date)
        for date in data2.keys():
            if date not in data1.keys():
                print("第一源比第二源 缺少日期", date)
        print("检查数据偏差")
        for date, v in data1.items():
            if date in data2.keys():
                v1 = data2[date]
                if isinstance(v1, dict):
                    nokey = []
                    for k, vv in v.items():
                        if isinstance(vv, list):
                            vv1 = v1[k]
                            vv.sort()
                            vv1.sort()
                            if vv != vv1:
                                print("第二源与第一源 数据异常", date, k, vv, vv1)
                        else:
                            if k not in nokey:
                                if k not in v1.keys():
                                    nokey.append(k)
                                    print("第二源比第一源 缺少字段值", date, k)
                                else:
                                    if vv != v1[k]:
                                        print("第二源与第一源 数据异常", date, k, vv, v1[k])
                elif isinstance(v1, list):
                    v.sort()
                    v1.sort()
                    if v != v1:
                        print("第二源与第一源 数据异常", k, v, v1)
                else:
                    if v != v1:
                        print("第二源与第一源 数据异常", k, v, v1)

    def get_maxtimekey(self, pcode=None, url=None, freq="%Y%m"):
        pcode, url, freq = self.get_arg(pcode, url, freq)
        base_url = os.path.join(url, pcode)
        date0=None
        if os.path.exists(base_url):
            l=os.listdir(base_url)
            if l:
                l.sort()
                a=load(os.path.join(base_url,l[-1]))
                date0=max(list(a.keys()))
        return date0

    def iter_data(self, start=None, end=None, pcode=None, url=None, freq="%Y%m"):
        #二次指定路径，因子名字
        pcode, url, freq = self.get_arg(pcode, url, freq)
        base_url = os.path.join(url, pcode)
        start_m=None
        end_m=None
        if start:
            start=str2datetime(start)
            start_m=start.strftime(freq)
        if end:
            end = str2datetime(end)
            end_m = end.strftime(freq)
        if os.path.exists(base_url):
            l=os.listdir(base_url)
            l.sort()
            for i in l:
                d0=i.split(".")[0].split("_")[-1]
                if start_m:
                    if d0<start_m:
                        continue
                if end_m:
                    if d0>end_m:
                        break
                kk = load(os.path.join(base_url, i))
                if kk   is None:
                    # print("不存在因子", pcode, os.path.join(url, pcode))
                    print("不存在因子", pcode, os.path.join(base_url, i))
                else:
                    for k, v in kk.items():
                        if start:
                            if k<start:
                                continue
                        if end:
                            if k>end:
                                break
                        yield k, v

    def get_bydate(self, date, pcode=None, url=None, freq="%Y%m"):
        pcode, url, freq = self.get_arg(pcode, url, freq)
        # base_url = os.path.join(url, pcode)
        d=None
        if pcode is None:
            if date  in self.data.keys():
                d= self.data[date]
        if d is None:
            key = date.strftime(freq)
            kk = load(os.path.join(url, pcode, pcode+"_"+key + ".pkl"))
            if kk:
                if pcode   is None:
                    self.data=kk
                if date in kk.keys():
                    d = kk[date]
        return d


class factordata:
    """
    0.可作为因子实例，也可以作为因子获取方法
    1.当作为因子实例时支持缓存，当作为 因子方法时不支持缓存
    2.自动判断是否按日志还是全部数据，优先使用按日期存储的数据
    3.支持因子保存功能，save方法，已经有全部转日志的方法
    """
    def __init__(self, name="", url=r"G:\factor\k1d", freq="%Y%m"):
        self.name = name
        self.url = url
        self.data = {}
        self.freq = freq
    def get_arg(self,name,url,freq):
        if name is None:
            name=self.name
        if url is None:
            url=self.url
        if freq is None:
            freq=self.freq
        return name,url,freq
    def deal2bydate(self, url, factors, freq="%Y%m", isfresh=False):
        for i in tqdm(factors):
            furl = os.path.join(url, i)
            print(furl)
            a = load(furl)
            if a:
                data = {}
                for date, v in a.items():
                    key = date.strftime(freq)
                    if key not in data.keys():
                        data[key] = {}
                    data[key][date] = v
                for key, v in data.items():
                    save_url = os.path.join(url, "bydate", i, key + ".pkl")
                    if not isfresh and os.path.exists(save_url):
                        continue
                    save(v, save_url)
    def save_all(self,data,name=None,url=None,freq="%Y%m",islog=True):
        name, url, freq = self.get_arg(name, url, freq)
        if data:
            data0 = {}
            for date, v in data.items():
                key = date.strftime(freq)
                if key not in data0.keys():
                    data0[key] = {}
                data0[key][date] = v
            for key, v in data0.items():
                save_url = os.path.join(url, "bydate", name, key + ".pkl")
                if os.path.exists(save_url):
                    vv=load(save_url)
                    vv.update(v)
                    save(vv, save_url)
                else:
                    save(v, save_url)

    def save_bydate(self,date,data,name=None,url=None,freq="%Y%m"):
        name, url, freq = self.get_arg(name, url, freq)
        url0 = os.path.join(url, name + ".pkl")
        if os.path.exists(url0):
            a = load(url0)
            a[date]=data
            save(a, url0)
        key = date.strftime(freq)
        save_url = os.path.join(url, "bydate", name, key + ".pkl")
        data1={}
        data1[date]=data
        if os.path.exists(save_url):
            a = load(save_url)
            a.update(data)
            save(a, save_url)
        else:
            save(data1, save_url)

    def save_new_factordata(self, data,name=None, url=None, freq="%Y%m"):
        name, url, freq = self.get_arg(name, url, freq)
        if data:
            url0 = os.path.join(url, name + ".pkl")
            if os.path.exists(url0):
                a = load(url0)
                a.update(data)
                save(a, url0)
            data1 = {}
            for date, v in data.items():
                key = date.strftime(freq)
                if key not in data1.keys():
                    data1[key] = {}
                data1[key][date] = v
            for key, v in data1.items():
                save_url = os.path.join(url, "bydate", name, key + ".pkl")
                if os.path.exists(save_url):
                    a = load(save_url)
                    a.update(v)
                    save(a, save_url)
                else:
                    save(v, save_url)
    def deal_byobj2bydate(self,data):
        rst={}
        for obj, v in data.items():
            for d, vv in v.items():
                if d not in rst.keys():
                    rst[d]={}
                rst[d][obj]=vv
        return rst
    def get_all(self,name=None,url=None,freq=None,start=None,end=None):
        name,url,freq=self.get_arg(name,url,freq)
        d={}
        if name is None:#判断是否缓存
            if self.data:
                d=self.data
        if not d:
            base_url = os.path.join(url, "bydate", name)
            if not os.path.exists(base_url):
                base_url0 = os.path.join(url, name+".pkl")
                if os.path.exists(base_url0):
                    self.deal2bydate(url, [name], freq=freq)
                    d=load(os.path.join(self.url, name))
            else:
                l=os.listdir(base_url)
                l.sort()
                for i in l:
                    isload=True
                    if start:
                        datestr = start.strftime(freq)
                        k=datestr+".pkl"
                        if i<k:
                            continue
                        if i==k:
                            data_url = os.path.join(base_url, i)
                            da0=load(data_url)
                            isload=False
                            for dd,v in da0.items():
                                if dd>=start:
                                    d[dd]=v
                    if end:
                        datestr = end.strftime(freq)
                        k = datestr + ".pkl"
                        if i > k:
                            continue
                        if i == k:
                            data_url = os.path.join(base_url, i)
                            da0 = load(data_url)
                            isload = False
                            for dd, v in da0.items():
                                if dd <= end:
                                    d[dd] = v
                    if isload:
                        data_url=os.path.join(base_url,i)
                        d.update(load(data_url))
            if name is None:
                self.data=d
        return  d

    def check_data(self, url1, url2, start_date, end_date):
        print(url1, url2, start_date, end_date)
        data1 = {}
        if url1[-4:] == ".pkl":
            data1 = load(url1)
        else:
            for date, v in self.iter_data(start=start_date, end=end_date, url=url1):
                data1[date] = v
        data2 = {}
        if url2[-4:] == ".pkl":
            data2 = load(url2)
        else:
            for date, v in self.iter_data(start=start_date, end=end_date, url=url2):
                data2[date] = v
        print("检查是否缺失数据,data1",len(data1),"data2",len(data2))
        if not data1:
            print("数据源一，没有数据", data1)
            return False
        if not data2:
            print("数据源二，没有数据", data2)
            return False
        for date in data1.keys():
            if date not in data2.keys():
                print("第二源比第一源 缺少日期", date)
        for date in data2.keys():
            if date not in data1.keys():
                print("第一源比第二源 缺少日期", date)
        print("检查数据偏差")
        for date, v in data1.items():
            if date in data2.keys():
                v1 = data2[date]
                if isinstance(v1, dict):
                    nokey = []
                    for k, vv in v.items():
                        if isinstance(vv, list):
                            vv1 = v1[k]
                            vv.sort()
                            vv1.sort()
                            if vv != vv1:
                                print("第二源与第一源 数据异常", date, k, vv, vv1)
                        else:
                            if k not in nokey:
                                if k not in v1.keys():
                                    nokey.append(k)
                                    print("第二源比第一源 缺少字段值", date, k)
                                else:
                                    if vv != v1[k]:
                                        print("第二源与第一源 数据异常", date, k, vv, v1[k])
                elif isinstance(v1, list):
                    v.sort()
                    v1.sort()
                    if v != v1:
                        print("第二源与第一源 数据异常", k, v, v1)
                else:
                    if v != v1:
                        print("第二源与第一源 数据异常", k, v, v1)

    def get_maxtimekey(self, name=None, url=None, freq="%Y%m"):
        name, url, freq = self.get_arg(name, url, freq)
        base_url = os.path.join(url, "bydate", name)
        date0=None
        if not os.path.exists(base_url):
            base_url0 = os.path.join(url, name + ".pkl")
            if os.path.exists(base_url0):
                self.deal2bydate(url, [name], freq=freq)
        if os.path.exists(base_url):
            l=os.listdir(base_url)
            ll=[]
            for i in l:
                ll.append(i.split(".")[0])
            ll.sort()
            if ll:
                d=max(ll)
                a=load(os.path.join(base_url,d))
                date0=max(list(a.keys()))
        return date0

    def iter_data(self, start=None, end=None, name=None, url=None, freq="%Y%m"):
        #二次指定路径，因子名字
        name, url, freq = self.get_arg(name, url, freq)
        base_url = os.path.join(url, "bydate", name)
        if not os.path.exists(base_url):
            base_url0 = os.path.join(url, name + ".pkl")
            if os.path.exists(base_url0):
                self.deal2bydate(url, [name], freq=freq)
        start_m=None
        end_m=None
        if start:
            start=str2datetime(start)
            start_m=start.strftime(freq)
        if end:
            end = str2datetime(end)
            end_m = end.strftime(freq)
        if os.path.exists(base_url):
            l=os.listdir(base_url)
            l.sort()
            for i in l:
                d0=i.split(".")[0]
                if start_m:
                    if d0<start_m:
                        continue
                if end_m:
                    if d0>end_m:
                        break
                kk = load(os.path.join(base_url, i))
                if kk   is None:
                    # print("不存在因子", name, os.path.join(url, name))
                    print("不存在因子", name, os.path.join(base_url, i))
                else:
                    for k, v in kk.items():
                        if start:
                            if k<start:
                                continue
                        if end:
                            if k>end:
                                break
                        yield k, v

    def get_bydate(self, date, start=None, end=None, name=None, url=None, freq="%Y%m"):
        d=None
        if name is None:
            if date in self.data.keys():
                d= self.data[date]
                return d
            datestr = date.strftime(freq)
            if datestr in self.data.keys():
                return self.data[datestr]
        name, url, freq = self.get_arg(name, url, freq)
        # base_url = os.path.join(url, "bydate", name)
        # if not os.path.exists(base_url):
        #     base_url0 = os.path.join(url, name + ".pkl")
        #     if os.path.exists(base_url0):
        #         self.deal2bydate(url, [name], freq=freq)

        if d is None:
            datestr = date.strftime(freq)
            kk = load(os.path.join(url, "bydate", name, datestr))
            if kk:
                self.data=kk
                if date in self.data.keys():
                    d = self.data[date]
            else:
                self.data[datestr]={}
            #     for
            #     self.data
        if d is None:
            return {}
        else:
            return d

def check_factors(factors,url,freq="%Y%m"):
    f=factordata()
    r={}
    for name in factors:
        base_url = os.path.join(url, "bydate", name)
        if not os.path.exists(base_url):
            base_url0 = os.path.join(url, name + ".pkl")
            if os.path.exists(base_url0):
                f.deal2bydate(url, [name], freq=freq)
        if os.path.exists(base_url):
            l = os.listdir(base_url)
            ll = []
            for i in l:
                ll.append(i.split(".")[0])
            ll.sort()
            if ll:
                t={}
                d = min(ll)
                a = load(os.path.join(base_url, d))
                t["start_date"] = min(list(a.keys()))
                d = max(ll)
                a = load(os.path.join(base_url, d))
                t["end_date"]  = max(list(a.keys()))
                r[name]=t
            else:
                r[name] = {"info": "不存在"}
        else:
            r[name]={"info":"不存在"}
    return r
class factor_ta:
    def __init__(self, args={}, volume0="nodrop", iscutpctCh=True, dataurl=r"G:\data\stock\hfq\byobj\k1d"):
        self.setargs(args)
        self.data = {}
        self.dataurl = dataurl
        self.volume0 = volume0
        self.iscutpctCh = iscutpctCh
        self.freq="%Y%m"
        # self.name=name

    def getargsout(self, arg):
        if "out" in arg.keys():
            return arg["out"]
        elif "arg" in arg.keys():
            if "out" in arg["arg"].keys():
                return arg["arg"]["out"]
            else:
                print("getargsout err")
                return []
        else:
            print("getargsout err")
            return []

    def setargs(self, args):
        self.args = args
        """
                if args:
            if isinstance(self.args, list):
                if isinstance(args, list):
                    self.args = args
                elif isinstance(args, dict):
                    t = []
                    if "calc_cmd" in args.keys():
                        argsid = args["calc_cmd"]
                        for i in self.args:
                            claccmd = i["calc_cmd"]
                            if argsid == claccmd:
                                i.update(args)
                            t.append(i)
                    self.args = t
            elif isinstance(self.args, dict):
                self.args.update(args)
        :param args: 
        :return: 
        """

    def update(self, facotr_url, k1d_url, batch_args, freq="%Y%m", start_date=None):
        self.freq =freq
        from jili.price.ta import creat_ta_calcor
        from jili.data.db import tradecal as tcl
        from jili.data.stocklocal import iter_bardict
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        url_byobj = os.path.join(kbase_url, "byobj")
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        datak = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(datak.keys()))
        for name, arg in batch_args.items():
            print("update", name)
            base_url = os.path.join(facotr_url, "bydate", name)
            base_url1 = os.path.join(facotr_url, name + ".pkl")
            isadd = True
            if not os.path.exists(base_url):
                if not os.path.exists(base_url1):
                    isadd = False
                    print("因子不存在,初始计算", name)
                    self.calc_history_batch(facotr_url, url_byobj, {name: arg}, refresh=False, ispool=True)
                else:
                    print("因子存在,按日期清洗", freq, name)
                factort = factordata("test")
                factort.deal2bydate(facotr_url, [name])
            if isadd:
                if start_date:
                    maxdate = str2datetime(start_date)
                else:
                    l = os.listdir(base_url)
                    l.sort()
                    data = load(os.path.join(base_url, l[-1]))
                    maxdate = max(list(data.keys()))

                if maxdate == maxdate_k:
                    print(name, "k线与因子已经是最新", maxdate)
                    continue
                data={}
                backdays = arg["batch"] + 3
                startdate = tcl.get_diff_tradeday(maxdate, -backdays)
                print("增量计算", maxdate, startdate, "回溯天数", backdays, "batch", arg["batch"])
                ta_factors = {}
                for date, bars in iter_bardict(startdate, url=k1d_url, ktype="k1d"):
                    for obj, bar in bars.items():
                        if obj not in ta_factors.keys():
                            ta_factors[obj] = creat_ta_calcor(arg)
                        t = ta_factors[obj].onbar(bar)
                        if len(t) == 1:
                            if date >= maxdate:
                                out = arg["out"][0]
                                tt = t[out]
                                if date not in data.keys():
                                    data[date] = {}
                                data[date][obj] = tt
                        else:
                            if date >= maxdate:
                                if date not in data.keys():
                                    data[date] = {}
                                data[date][obj] = t
                data1 = {}
                for date, v in data.items():
                    key = date.strftime(freq)
                    if key not in data1.keys():
                        data1[key] = {}
                    data1[key][date] = v
                for key, v in data1.items():
                    save_url = os.path.join(facotr_url, "bydate", name, key + ".pkl")
                    if os.path.exists(save_url):
                        a = load(save_url)
                        a.update(v)
                        save(a, save_url)
                    else:
                        save(v, save_url)

    def getargs(self):
        return self.args

    def calc_graph(self, k, args):
        data = {}
        if isinstance(k, list):
            kds = pd.DataFrame(k)
        else:
            kds = k
        objs = kds["obj"].drop_duplicates().to_list()
        if len(objs) == 1:
            obj = objs[0]
            kd = kds
            kd.index = kd["timekey"]
            kd.sort_index(inplace=True)
            if self.iscutpctCh and "pctChg" in kd.columns:
                if obj[0] == "3":
                    kd.loc[(kd.index >= "20200427") & (kd["pctChg"] > 20), "pctChg"] = 20
                    kd.loc[(kd.index < "20200427") & (kd["pctChg"] > 10), "pctChg"] = 10
                elif obj[:3] == "688":
                    kd.loc[kd["pctChg"] > 20, "pctChg"] = 20
                else:
                    kd.loc[kd["pctChg"] > 10, "pctChg"] = 10
            if self.volume0 == "drop":
                kd1 = kd[kd["volume"] > 0]
                if self.iscalc(kd, args):
                    if isinstance(args, list):
                        kd1 = graph.run(kd1, args)
                        out = self.getargsout(args[-1])
                    else:
                        out = self.getargsout(args)
                        kd1 = graph.run(kd1, [args])
                else:
                    if isinstance(args, list):
                        out = self.getargsout(args[-1])
                    else:
                        out = self.getargsout(args)
                    for i in out:
                        kd1[i] = np.nan
                kd = kd1.reindex(kd.index)
                kd["obj"] = obj
                kd["timekey"] = kd.index
            else:
                if self.iscalc(kd, args):
                    if isinstance(args, list):
                        kd = graph.run(kd, args)
                        out = self.getargsout(args[-1])
                    else:
                        out = self.getargsout(args)
                        kd = graph.run(kd, [args])
                else:
                    if isinstance(args, list):
                        out = self.getargsout(args[-1])
                    else:
                        out = self.getargsout(args)
                    for i in out:
                        kd[i] = np.nan
            for k1, v1 in kd[out].to_dict("index").items():
                if isinstance(k1, pd.Timestamp):
                    k1 = k1.to_pydatetime()
                if k1 not in data.keys():
                    t = {}
                    t[obj] = v1
                    data[k1] = t
                else:
                    t = data[k1]
                    t[obj] = v1
        else:
            for obj in tqdm(objs):
                kd = kds.loc[kds[obj] == obj]
                kd.index = kd["timekey"]
                kd.sort_index(inplace=True)
                if self.iscutpctCh and "pctChg" in kd.columns:
                    if obj[0] == "3":
                        kd.loc[(kd.index >= "20200427") & (kd["pctChg"] > 20), "pctChg"] = 20
                        kd.loc[(kd.index < "20200427") & (kd["pctChg"] > 10), "pctChg"] = 10
                    elif obj[:3] == "688":
                        kd.loc[kd["pctChg"] > 20, "pctChg"] = 20
                    else:
                        kd.loc[kd["pctChg"] > 10, "pctChg"] = 10
                if self.volume0 == "drop":
                    kd1 = kd[kd["volume"] > 0]
                    if self.iscalc(kd, args):
                        if isinstance(args, list):
                            kd1 = graph.run(kd1, args)
                            out = self.getargsout(args[-1])
                        else:
                            out = self.getargsout(args)
                            kd1 = graph.run(kd1, [args])
                    else:
                        if isinstance(args, list):
                            out = self.getargsout(args[-1])
                        else:
                            out = self.getargsout(args)
                        for i in out:
                            kd1[i] = np.nan
                    kd = kd1.reindex(kd.index)
                    kd["obj"] = obj
                    kd["timekey"] = kd.index
                else:
                    if self.iscalc(kd, args):
                        if isinstance(args, list):
                            kd = graph.run(kd, args)
                            out = self.getargsout(args[-1])
                        else:
                            out = self.getargsout(args)
                            kd = graph.run(kd, [args])
                    else:
                        if isinstance(args, list):
                            out = self.getargsout(args[-1])
                        else:
                            out = self.getargsout(args)
                        for i in out:
                            kd[i] = np.nan
                for k1, v1 in kd[out].to_dict("index").items():
                    if isinstance(k1, pd.Timestamp):
                        k1 = k1.to_pydatetime()
                    if k1 not in data.keys():
                        t = {}
                        t[obj] = v1
                        data[k1] = t
                    else:
                        t = data[k1]
                        t[obj] = v1
        return data

    def calc(self):
        data = {}
        for i in tqdm(os.listdir(self.dataurl)):
            obj = i.split("_")[0]
            k = load(os.path.join(self.dataurl, i))
            kd = pd.DataFrame(k)
            kd.index = kd["timekey"]
            kd.sort_index(inplace=True)
            if self.iscutpctCh and "pctChg" in kd.columns:
                if obj[0] == "3":
                    kd.loc[(kd.index >= "20200427") & (kd["pctChg"] > 20), "pctChg"] = 20
                    kd.loc[(kd.index < "20200427") & (kd["pctChg"] > 10), "pctChg"] = 10
                elif obj[:3] == "688":
                    kd.loc[kd["pctChg"] > 20, "pctChg"] = 20
                else:
                    kd.loc[kd["pctChg"] > 10, "pctChg"] = 10
            if self.volume0 == "drop":
                kd1 = kd[kd["volume"] > 0]
                if self.iscalc(kd, self.args):
                    if isinstance(self.args, list):
                        kd1 = graph.run(kd1, self.args)
                        out = self.getargsout(self.args[-1])
                    else:
                        out = self.getargsout(self.args)
                        kd1 = graph.run(kd1, [self.args])
                else:
                    if isinstance(self.args, list):
                        out = self.getargsout(self.args[-1])
                    else:
                        out = self.getargsout(self.args)
                    for i in out:
                        kd1[i] = np.nan
                kd = kd1.reindex(kd.index)
                kd["obj"] = obj
                kd["timekey"] = kd.index
            else:
                if self.iscalc(kd, self.args):
                    if isinstance(self.args, list):
                        kd = graph.run(kd, self.args)
                        out = self.getargsout(self.args[-1])
                    else:
                        out = self.getargsout(self.args)
                        kd = graph.run(kd, [self.args])
                else:
                    if isinstance(self.args, list):
                        out = self.getargsout(self.args[-1])
                    else:
                        out = self.getargsout(self.args)
                    for i in out:
                        kd[i] = np.nan
            for k1, v1 in kd[out].to_dict("index").items():
                if isinstance(k1, pd.Timestamp):
                    k1 = k1.to_pydatetime()
                if k1 not in data.keys():
                    t = {}
                    t[obj] = v1
                    data[k1] = t
                else:
                    t = data[k1]
                    t[obj] = v1
        return data

    def iscalc(self, k, args):
        t = args.get("batch", 1)
        return len(k) >= t

    def calc_history_batch(self, location=r"G:/factor", k1d_url=None, batch_args=None, refresh=False, ispool=True):
        batch_args0 = {}
        for name, arg in batch_args.items():
            # base_url = os.path.join(location, name + ".pkl")
            # if not refresh and os.path.exists(base_url):
            #     pass
            # else:
            batch_args0[name] = arg
        if k1d_url:
            self.dataurl = k1d_url
        if ispool:
            import multiprocessing
            pool = multiprocessing.Pool(multiprocessing.cpu_count() - config.cpu_null_n)
            for i in os.listdir(self.dataurl):
                pool.apply_async(self.calc_batch_save, (i, batch_args0, location,))
            pool.close()
            pool.join()
            pool = multiprocessing.Pool(multiprocessing.cpu_count() - config.cpu_null_n)
            for name, args in batch_args.items():
                pool.apply_async(self.redata, (name, location,))
            pool.close()
            pool.join()
        else:
            l = os.listdir(self.dataurl)
            for i in tqdm(l):
                self.calc_batch_save(i, batch_args0, location)
            for name, args in batch_args.items():
                self.redata(name, location)

    def redata(self, name, location):
        data = {}
        url = os.path.join(location, name)
        for i in os.listdir(url):
            k = load(os.path.join(location, name, i))
            for date, v in k.items():
                if date not in data.keys():
                    data[date] = {}
                data[date].update(v)
        data1 = {}
        for date, v in data.items():
            key = date.strftime(self.freq)
            if key not in data1.keys():
                data1[key] = {}
            data1[key][date] = v
        for key, v in data1.items():
            save_url = os.path.join(location, "bydate", name, key + ".pkl")
            if os.path.exists(save_url):
                a = load(save_url)
                a.update(v)
                save(a, save_url)
            else:
                save(v, save_url)
        print("redata", len(data), name)
        for i in os.listdir(url):
            os.remove(os.path.join(location, name, i))
        os.rmdir(url)

    def calc_batch_save(self, i, batch_args, location):
        obj = i.split("_")[0]
        k = load(os.path.join(self.dataurl, i))
        if k:
            kd = pd.DataFrame(k)
            kd.index = kd["timekey"]
            kd.sort_index(inplace=True)
            if self.iscutpctCh and "pctChg" in kd.columns:
                if obj[0] == "3":
                    kd.loc[(kd.index >= "20200427") & (kd["pctChg"] > 20), "pctChg"] = 20
                    kd.loc[(kd.index < "20200427") & (kd["pctChg"] > 10), "pctChg"] = 10
                elif obj[:3] == "688":
                    kd.loc[kd["pctChg"] > 20, "pctChg"] = 20
                else:
                    kd.loc[kd["pctChg"] > 10, "pctChg"] = 10
            outs = {}
            if self.volume0 == "drop":
                kd1 = kd[kd["volume"] > 0]
                for name, args in batch_args.items():
                    if self.iscalc(kd, args):
                        if isinstance(args, list):
                            kd1 = graph.run(kd1, args)
                            out = self.getargsout(args[-1])
                        else:
                            out = self.getargsout(args)
                            kd1 = graph.run(kd1, [args])
                    else:
                        if isinstance(args, list):
                            out = self.getargsout(args[-1])
                        else:
                            out = self.getargsout(args)
                        for i in out:
                            kd1[i] = np.nan

                    outs[name] = out
                kd = kd1.reindex(kd.index)
                kd["obj"] = obj
                kd["timekey"] = kd.index
            else:
                for name, args in batch_args.items():
                    if self.iscalc(kd, args):
                        if isinstance(args, list):
                            kd = graph.run(kd, args)
                            out = self.getargsout(args[-1])
                        else:
                            out = self.getargsout(args)
                            kd = graph.run(kd, [args])
                    else:
                        if isinstance(args, list):
                            out = self.getargsout(args[-1])
                        else:
                            out = self.getargsout(args)
                        for i in out:
                            kd[i] = np.nan
                    outs[name] = out
            for name, out in outs.items():
                data = {}
                if len(out) > 1:
                    for k1, v1 in kd[out].to_dict("index").items():
                        if isinstance(k1, pd.Timestamp):
                            k1 = k1.to_pydatetime()
                        data[k1] = {obj: v1}
                else:
                    o = out[0]
                    for k1, v1 in kd[o].to_dict("index").items():
                        if isinstance(k1, pd.Timestamp):
                            k1 = k1.to_pydatetime()
                        data[k1] = {obj: v1}
                url0 = os.path.join(location, name, obj + ".pkl")
                save(data, url0)

    def get_history(self, source="pkl", location=r"G:/factor", name="name", args=None, refresh=False):
        if args:
            self.setargs(args)
        if source == "pkl":
            base_url = os.path.join(location, name + ".pkl")
            if refresh and os.path.exists(base_url):
                return load(base_url)
            else:
                data = self.calc()
                save(data, base_url)
                return data
        else:
            if location:
                db = getdb_client("factor", location)
            else:
                db = getdb_client("factor")
            data = {}
            cu = db[name]
            for i in cu.find({}):
                del i["_id"]
                tk = i["timekey"]
                obj = i["obj"]
                if tk not in data.keys():
                    t = {}
                    t[obj] = i
                    data[tk] = t
                else:
                    t = data[tk]
                    t[obj] = i
            if data:
                return data
            else:
                data = self.calc()
                cu.create_index([("timekey", 1), ("obj", 1)], unique=True, background=True)
                cu.create_index([("timekey", 1)], unique=False, background=True)
                cu.create_index([("obj", 1)], unique=False, background=True)
                for v in data.values():
                    for i in v.values():
                        insert_one(cu, i)
                return data

    def get_history(self, source="pkl", location=r"G:/factor", name="name", args=None):
        if args:
            self.setargs(args)
        if source == "pkl":
            base_url = os.path.join(location, name)
            if os.path.exists(base_url):
                return load(base_url)
            else:
                data = self.calc()
                save(data, base_url)
                return data
        else:
            if location:
                db = getdb_client("factor", location)
            else:
                db = getdb_client("factor")
            data = {}
            cu = db[name]
            for i in cu.find({}):
                del i["_id"]
                tk = i["timekey"]
                obj = i["obj"]
                if tk not in data.keys():
                    t = {}
                    t[obj] = i
                    data[tk] = t
                else:
                    t = data[tk]
                    t[obj] = i
            if data:
                return data
            else:
                data = self.calc()
                cu.create_index([("timekey", 1), ("obj", 1)], unique=True, background=True)
                cu.create_index([("timekey", 1)], unique=False, background=True)
                cu.create_index([("obj", 1)], unique=False, background=True)
                for v in data.values():
                    for i in v.values():
                        insert_one(cu, i)
                return data

    def save(self, source="pkl", location=r"G:/factor", name="name"):
        pass

    def get_last(self, source="pkl", location=r"G:/factor", name="name"):
        pass

    def get_bydate(self, source="pkl", location=r"G:/factor", name="name"):
        pass


if __name__ == '__main__':
    f = factordata_f(pcode="TA", url=r"D:\factor\k1d")
    import datetime

    data = f.get_bydate(datetime.datetime(2022, 11, 29))
    print("end")


    # factor_url = r"G:\factor\k1d"
    # k1d_url = r"G:\data\stock"
    # date = "20220601"
    # a = factor_ta(iscutpctCh=False)
    # batchargs = {'isST': {'calc_cmd': 'ta', 'cmd': 'MEDPRICE', 'out': ['isST'],
    #              'input': {'prices': ['high', 'low']}, 'arg': {}, 'batch': 1},}
    # a.update(factor_url, k1d_url=k1d_url, batch_args=batchargs, freq="%Y%m", start_date=date)
    # f4 = factordata(name="jg_hold_info")
    # top10_holders_jgrate = {}
    # import datetime
    # for k, v in f4.iter_data(datetime.datetime(2022, 6, 14), datetime.datetime(2022, 6, 20)):
    #     top10_holders_jgrate[k] = v
    # a=factor_hisbatch_bygrap()
    # print(a.dataurl)
    # a=factordata("test")
    # a.deal2bydate(r"G:\factor\k1d",["is_wind_concept_detail_l1","is_wind_detail_l1","jg_hold_info"])
    #    a.deal2bydate(r"G:\factor\k1d_0922\new1", ["ema_close_5", "ema_close_10", "ema_close_20","ema_close_40","ema_close_60","ema_close_120",
    #                                          "future_minmax_5","future_minmax_10","future_minmax_20","future_minmax_30","future_minmax_40","future_minmax_60","future_minmax_90","future_minmax_120",
    #                                          "future_pl_5","future_pl_10","future_pl_20","future_pl_30","future_pl_40","future_pl_60","future_pl_90","future_pl_120",
    #                                          "ma_amount_10","ma_amount_20","ma_close_5","ma_close_10","ma_close_20","ma_close_40","ma_close_60","ma_close_120",
    #                                          "finance_info","is_dc_detail_l1","is_dc_detail_l2","is_st","is_sw_detail_l1","is_sw_detail_l2","ma_turn_5","ma_turn_10",
    #                                          "ma_volume_5","ma_volume_10","ma_volume_20","new_high_days","new_high_days_byhigh","new_low_days","new_low_days_bylow",
    #                                          "pctchg_2","pctchg_3","pctchg_10","pctchg_20","pctchg_30","pctchg_40","pctchg_50","pctchg_60","pctchg_90","pctchg_120",
    #                                          "rps_10","rps_20","rps_40","rps_50","rps_60","rps_90","rps_120","size_info","top_inst_jg_buy","top_inst_jg_sell",
    #                                          "trade_days","trade_info"])

    #    a.deal2bydate(r"G:\factor\k1d", [
    #                     "is_st", "ma_amount_20", "jg_hold_info",
    #                     "is_new_high_5", "trade_days",
    #                     "pctchg_2", "pctchg_3", "pctchg_10", "pctchg_20", "pctchg_50", "rps_50", "rps_20",
    #                     "trade_info", "finance_info", "size_info",
    #                     "is_dc_detail_l1", "is_dc_detail_l2",
    #                     ])
    #     a.deal2bydate(r"G:\factor\k1d", ['rps_10'])
    # a = factor_ta()
    # batchargs = {
    #     "ma_amount_20": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_amount_20'],
    #                      'input': {'price': 'amount'},
    #                      'arg': {'timeperiod': 20, 'matype': 0}, 'batch': 20},
    #     "pctchg_2": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_2'],
    #                  'input': {'price': 'pctChg'},
    #                  'arg': {'timeperiod': 2}, 'batch': 2},
    #     "pctchg_3": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_3'],
    #                  'input': {'price': 'pctChg'},
    #                  'arg': {'timeperiod': 3}, 'batch': 3},
    #     "pctchg_4": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_4'],
    #                  'input': {'price': 'pctChg'},
    #                  'arg': {'timeperiod': 4}, 'batch': 4},
    #     "pctchg_5": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_5'],
    #                  'input': {'price': 'pctChg'},
    #                  'arg': {'timeperiod': 5}, 'batch': 5},
    #     "pctchg_6": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_6'],
    #                  'input': {'price': 'pctChg'},
    #                  'arg': {'timeperiod': 6}, 'batch': 6},
    #     'tech_high_max60': {'calc_cmd': 'ta', 'cmd': 'MAX', 'out': ['tech_high_max60'],
    #                         'input': {'price': 'high'},
    #                         'arg': {'timeperiod': 60}, 'batch': 60},
    #     'tech_high': {'calc_cmd': 'ta', 'cmd': 'MEDPRICE', 'out': ['tech_high'],
    #                   'input': {'prices': ['high', 'high']}, 'arg': {}, 'batch': 1},
    #     'tech_close': {'calc_cmd': 'ta', 'cmd': 'MEDPRICE', 'out': ['tech_close'],
    #                    'input': {'prices': ['close', 'close']}, 'arg': {}, 'batch': 1},
    #     "ma_close_10": {'calc_cmd': 'ta', 'cmd': 'MA', 'out': ['ma_close_10'],
    #                     'input': {'price': 'close'},
    #                     'arg': {'timeperiod': 10, 'matype': 0}, 'batch': 10},
    # }
    #
    # a.update(r"G:\factor\test", k1d_url=r"G:\data\stock", batch_args=batchargs, freq="%Y%m", start_date="20210801")